/*
 *   This program is free software: you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation, either version 3 of the License, or
 *   (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

/*
 *    ClassifierSplitEvaluator.java
 *    Copyright (C) 1999-2012 University of Waikato, Hamilton, New Zealand
 *
 */

package weka.experiment;

import java.io.ByteArrayOutputStream;
import java.io.ObjectOutputStream;
import java.io.ObjectStreamClass;
import java.io.Serializable;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Enumeration;
import java.util.List;
import java.util.Vector;

import weka.classifiers.AbstractClassifier;
import weka.classifiers.Classifier;
import weka.classifiers.Evaluation;
import weka.classifiers.evaluation.AbstractEvaluationMetric;
import weka.classifiers.rules.ZeroR;
import weka.core.AdditionalMeasureProducer;
import weka.core.Attribute;
import weka.core.Instances;
import weka.core.Option;
import weka.core.OptionHandler;
import weka.core.Summarizable;
import weka.core.Utils;

/**
 * <!-- globalinfo-start --> A SplitEvaluator that produces results for a
 * classification scheme on a nominal class attribute.
 * <p/>
 * <!-- globalinfo-end -->
 *
 * <!-- options-start --> Valid options are:
 * <p/>
 *
 * <pre>
 * -W &lt;class name&gt;
 *  The full class name of the classifier.
 *  eg: weka.classifiers.bayes.NaiveBayes
 * </pre>
 *
 * <pre>
 * -C &lt;index&gt;
 *  The index of the class for which IR statistics
 *  are to be output. (default 1)
 * </pre>
 *
 * <pre>
 * -I &lt;index&gt;
 *  The index of an attribute to output in the
 *  results. This attribute should identify an
 *  instance in order to know which instances are
 *  in the test set of a cross validation. if 0
 *  no output (default 0).
 * </pre>
 *
 * <pre>
 * -P
 *  Add target and prediction columns to the result
 *  for each fold.
 * </pre>
 *
 * <pre>
 * -no-size
 *  Skips the determination of sizes (train/test/classifier)
 *  (default: sizes are determined)
 * </pre>
 *
 * <pre>
 * Options specific to classifier weka.classifiers.rules.ZeroR:
 * </pre>
 *
 * <pre>
 * -D
 *  If set, classifier is run in debug mode and
 *  may output additional info to the console
 * </pre>
 *
 * <!-- options-end -->
 *
 * All options after -- will be passed to the classifier.
 *
 * @author Len Trigg (trigg@cs.waikato.ac.nz)
 * @version $Revision$
 */
public class ClassifierSplitEvaluator implements SplitEvaluator, OptionHandler, AdditionalMeasureProducer {

    /** for serialization */
    static final long serialVersionUID = -8511241602760467265L;

    /** The template classifier */
    protected Classifier m_Template = new ZeroR();

    /** The classifier used for evaluation */
    protected Classifier m_Classifier;

    /** Holds the most recently used Evaluation object */
    protected Evaluation m_Evaluation;

    /** The names of any additional measures to look for in SplitEvaluators */
    protected String[] m_AdditionalMeasures = null;

    /**
     * Array of booleans corresponding to the measures in m_AdditionalMeasures
     * indicating which of the AdditionalMeasures the current classifier can produce
     */
    protected boolean[] m_doesProduce = null;

    /**
     * The number of additional measures that need to be filled in after taking into
     * account column constraints imposed by the final destination for results
     */
    protected int m_numberAdditionalMeasures = 0;

    /** Holds the statistics for the most recent application of the classifier */
    protected String m_result = null;

    /** The classifier options (if any) */
    protected String m_ClassifierOptions = "";

    /** The classifier version */
    protected String m_ClassifierVersion = "";

    /** The length of a key */
    private static final int KEY_SIZE = 3;

    /** The length of a result */
    private static final int RESULT_SIZE = 32;

    /** The number of IR statistics */
    private static final int NUM_IR_STATISTICS = 16;

    /** The number of averaged IR statistics */
    private static final int NUM_WEIGHTED_IR_STATISTICS = 10;

    /** The number of unweighted averaged IR statistics */
    private static final int NUM_UNWEIGHTED_IR_STATISTICS = 2;

    /** Class index for information retrieval statistics (default 0) */
    private int m_IRclass = 0;

    /** Flag for prediction and target columns output. */
    private boolean m_predTargetColumn = false;

    /** Attribute index of instance identifier (default -1) */
    private int m_attID = -1;

    /** whether to skip determination of sizes (train/test/classifier). */
    private boolean m_NoSizeDetermination;

    protected final List<AbstractEvaluationMetric> m_pluginMetrics = new ArrayList<AbstractEvaluationMetric>();
    protected int m_numPluginStatistics = 0;

    /**
     * No args constructor.
     */
    public ClassifierSplitEvaluator() {

        updateOptions();

        List<AbstractEvaluationMetric> pluginMetrics = AbstractEvaluationMetric.getPluginMetrics();
        if (pluginMetrics != null) {
            for (AbstractEvaluationMetric m : pluginMetrics) {
                System.err.println(m.getMetricName());
                if (m.appliesToNominalClass()) {
                    m_pluginMetrics.add(m);
                    m_numPluginStatistics += m.getStatisticNames().size();
                }
            }
        }
    }

    /**
     * Returns a string describing this split evaluator
     *
     * @return a description of the split evaluator suitable for displaying in the
     *         explorer/experimenter gui
     */
    public String globalInfo() {
        return " A SplitEvaluator that produces results for a classification " + "scheme on a nominal class attribute.";
    }

    /**
     * Returns an enumeration describing the available options..
     *
     * @return an enumeration of all the available options.
     */
    @Override
    public Enumeration<Option> listOptions() {

        Vector<Option> newVector = new Vector<Option>(5);

        newVector.addElement(new Option("\tThe full class name of the classifier.\n" + "\teg: weka.classifiers.bayes.NaiveBayes", "W", 1, "-W <class name>"));
        newVector.addElement(new Option("\tThe index of the class for which IR statistics\n" + "\tare to be output. (default 1)", "C", 1, "-C <index>"));
        newVector.addElement(new Option("\tThe index of an attribute to output in the\n" + "\tresults. This attribute should identify an\n" + "\tinstance in order to know which instances are\n" + "\tin the test set of a cross validation. if 0\n" + "\tno output (default 0).", "I", 1, "-I <index>"));
        newVector.addElement(new Option("\tAdd target and prediction columns to the result\n" + "\tfor each fold.", "P", 0, "-P"));
        newVector.addElement(new Option("\tSkips the determination of sizes (train/test/classifier)\n" + "\t(default: sizes are determined)", "no-size", 0, "-no-size"));

        if ((m_Template != null) && (m_Template instanceof OptionHandler)) {
            newVector.addElement(new Option("", "", 0, "\nOptions specific to classifier " + m_Template.getClass().getName() + ":"));
            newVector.addAll(Collections.list(((OptionHandler) m_Template).listOptions()));

        }
        return newVector.elements();
    }

    /**
     * Parses a given list of options.
     * <p/>
     *
     * <!-- options-start --> Valid options are:
     * <p/>
     *
     * <pre>
     * -W &lt;class name&gt;
     *  The full class name of the classifier.
     *  eg: weka.classifiers.bayes.NaiveBayes
     * </pre>
     *
     * <pre>
     * -C &lt;index&gt;
     *  The index of the class for which IR statistics
     *  are to be output. (default 1)
     * </pre>
     *
     * <pre>
     * -I &lt;index&gt;
     *  The index of an attribute to output in the
     *  results. This attribute should identify an
     *  instance in order to know which instances are
     *  in the test set of a cross validation. if 0
     *  no output (default 0).
     * </pre>
     *
     * <pre>
     * -P
     *  Add target and prediction columns to the result
     *  for each fold.
     * </pre>
     *
     * <pre>
     * -no-size
     *  Skips the determination of sizes (train/test/classifier)
     *  (default: sizes are determined)
     * </pre>
     *
     * <pre>
     * Options specific to classifier weka.classifiers.rules.ZeroR:
     * </pre>
     *
     * <pre>
     * -D
     *  If set, classifier is run in debug mode and
     *  may output additional info to the console
     * </pre>
     *
     * <!-- options-end -->
     *
     * All options after -- will be passed to the classifier.
     *
     * @param options the list of options as an array of strings
     * @throws Exception if an option is not supported
     */
    @Override
    public void setOptions(String[] options) throws Exception {

        String cName = Utils.getOption('W', options);
        if (cName.length() == 0) {
            throw new Exception("A classifier must be specified with" + " the -W option.");
        }
        // Do it first without options, so if an exception is thrown during
        // the option setting, listOptions will contain options for the actual
        // Classifier.
        setClassifier(AbstractClassifier.forName(cName, null));
        if (getClassifier() instanceof OptionHandler) {
            ((OptionHandler) getClassifier()).setOptions(Utils.partitionOptions(options));
            updateOptions();
        }

        String indexName = Utils.getOption('C', options);
        if (indexName.length() != 0) {
            m_IRclass = (new Integer(indexName)).intValue() - 1;
        } else {
            m_IRclass = 0;
        }

        String attID = Utils.getOption('I', options);
        if (attID.length() != 0) {
            m_attID = (new Integer(attID)).intValue() - 1;
        } else {
            m_attID = -1;
        }

        m_predTargetColumn = Utils.getFlag('P', options);
        m_NoSizeDetermination = Utils.getFlag("no-size", options);
    }

    /**
     * Gets the current settings of the Classifier.
     *
     * @return an array of strings suitable for passing to setOptions
     */
    @Override
    public String[] getOptions() {
        Vector<String> result;
        String[] classifierOptions;

        result = new Vector<String>();

        classifierOptions = new String[0];
        if ((m_Template != null) && (m_Template instanceof OptionHandler)) {
            classifierOptions = ((OptionHandler) m_Template).getOptions();
        }

        if (getClassifier() != null) {
            result.add("-W");
            result.add(getClassifier().getClass().getName());
        }
        result.add("-I");
        result.add("" + (m_attID + 1));

        if (getPredTargetColumn()) {
            result.add("-P");
        }

        result.add("-C");
        result.add("" + (m_IRclass + 1));

        if (getNoSizeDetermination()) {
            result.add("-no-size");
        }

        result.add("--");
        result.addAll(Arrays.asList(classifierOptions));

        return result.toArray(new String[result.size()]);
    }

    /**
     * Set a list of method names for additional measures to look for in
     * Classifiers. This could contain many measures (of which only a subset may be
     * produceable by the current Classifier) if an experiment is the type that
     * iterates over a set of properties.
     *
     * @param additionalMeasures a list of method names
     */
    @Override
    public void setAdditionalMeasures(String[] additionalMeasures) {
        // System.err.println("ClassifierSplitEvaluator: setting additional measures");
        m_AdditionalMeasures = additionalMeasures;

        // determine which (if any) of the additional measures this classifier
        // can produce
        if (m_AdditionalMeasures != null && m_AdditionalMeasures.length > 0) {
            m_doesProduce = new boolean[m_AdditionalMeasures.length];

            if (m_Template instanceof AdditionalMeasureProducer) {
                Enumeration<String> en = ((AdditionalMeasureProducer) m_Template).enumerateMeasures();
                while (en.hasMoreElements()) {
                    String mname = en.nextElement();
                    for (int j = 0; j < m_AdditionalMeasures.length; j++) {
                        if (mname.compareToIgnoreCase(m_AdditionalMeasures[j]) == 0) {
                            m_doesProduce[j] = true;
                        }
                    }
                }
            }
        } else {
            m_doesProduce = null;
        }
    }

    /**
     * Returns an enumeration of any additional measure names that might be in the
     * classifier
     *
     * @return an enumeration of the measure names
     */
    @Override
    public Enumeration<String> enumerateMeasures() {
        Vector<String> newVector = new Vector<String>();
        if (m_Template instanceof AdditionalMeasureProducer) {
            Enumeration<String> en = ((AdditionalMeasureProducer) m_Template).enumerateMeasures();
            while (en.hasMoreElements()) {
                String mname = en.nextElement();
                newVector.add(mname);
            }
        }
        return newVector.elements();
    }

    /**
     * Returns the value of the named measure
     *
     * @param additionalMeasureName the name of the measure to query for its value
     * @return the value of the named measure
     * @throws IllegalArgumentException if the named measure is not supported
     */
    @Override
    public double getMeasure(String additionalMeasureName) {
        if (m_Template instanceof AdditionalMeasureProducer) {
            if (m_Classifier == null) {
                throw new IllegalArgumentException("ClassifierSplitEvaluator: " + "Can't return result for measure, " + "classifier has not been built yet.");
            }
            return ((AdditionalMeasureProducer) m_Classifier).getMeasure(additionalMeasureName);
        } else {
            throw new IllegalArgumentException("ClassifierSplitEvaluator: " + "Can't return value for : " + additionalMeasureName + ". " + m_Template.getClass().getName() + " " + "is not an AdditionalMeasureProducer");
        }
    }

    /**
     * Gets the data types of each of the key columns produced for a single run. The
     * number of key fields must be constant for a given SplitEvaluator.
     *
     * @return an array containing objects of the type of each key column. The
     *         objects should be Strings, or Doubles.
     */
    @Override
    public Object[] getKeyTypes() {

        Object[] keyTypes = new Object[KEY_SIZE];
        keyTypes[0] = "";
        keyTypes[1] = "";
        keyTypes[2] = "";
        return keyTypes;
    }

    /**
     * Gets the names of each of the key columns produced for a single run. The
     * number of key fields must be constant for a given SplitEvaluator.
     *
     * @return an array containing the name of each key column
     */
    @Override
    public String[] getKeyNames() {

        String[] keyNames = new String[KEY_SIZE];
        keyNames[0] = "Scheme";
        keyNames[1] = "Scheme_options";
        keyNames[2] = "Scheme_version_ID";
        return keyNames;
    }

    /**
     * Gets the key describing the current SplitEvaluator. For example This may
     * contain the name of the classifier used for classifier predictive evaluation.
     * The number of key fields must be constant for a given SplitEvaluator.
     *
     * @return an array of objects containing the key.
     */
    @Override
    public Object[] getKey() {

        Object[] key = new Object[KEY_SIZE];
        key[0] = m_Template.getClass().getName();
        key[1] = m_ClassifierOptions;
        key[2] = m_ClassifierVersion;
        return key;
    }

    /**
     * Gets the data types of each of the result columns produced for a single run.
     * The number of result fields must be constant for a given SplitEvaluator.
     *
     * @return an array containing objects of the type of each result column. The
     *         objects should be Strings, or Doubles.
     */
    @Override
    public Object[] getResultTypes() {
        int addm = (m_AdditionalMeasures != null) ? m_AdditionalMeasures.length : 0;
        int overall_length = RESULT_SIZE + addm;
        overall_length += NUM_IR_STATISTICS;
        overall_length += NUM_WEIGHTED_IR_STATISTICS;
        overall_length += NUM_UNWEIGHTED_IR_STATISTICS;

        if (getAttributeID() >= 0) {
            overall_length += 1;
        }
        if (getPredTargetColumn()) {
            overall_length += 2;
        }

        overall_length += m_numPluginStatistics;

        Object[] resultTypes = new Object[overall_length];
        Double doub = new Double(0);
        int current = 0;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;

        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;

        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;

        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;

        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;

        // IR stats
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;

        // Unweighted IR stats
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;

        // Weighted IR stats
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;

        // Timing stats
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;

        // sizes
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;

        // Prediction interval statistics
        resultTypes[current++] = doub;
        resultTypes[current++] = doub;

        // ID/Targets/Predictions
        if (getAttributeID() >= 0) {
            resultTypes[current++] = "";
        }
        if (getPredTargetColumn()) {
            resultTypes[current++] = "";
            resultTypes[current++] = "";
        }

        // Classifier defined extras
        resultTypes[current++] = "";

        // add any additional measures
        for (int i = 0; i < addm; i++) {
            resultTypes[current++] = doub;
        }

        // plugin metrics
        for (int i = 0; i < m_numPluginStatistics; i++) {
            resultTypes[current++] = doub;
        }

        if (current != overall_length) {
            throw new Error("ResultTypes didn't fit RESULT_SIZE");
        }
        return resultTypes;
    }

    /**
     * Gets the names of each of the result columns produced for a single run. The
     * number of result fields must be constant for a given SplitEvaluator.
     *
     * @return an array containing the name of each result column
     */
    @Override
    public String[] getResultNames() {
        int addm = (m_AdditionalMeasures != null) ? m_AdditionalMeasures.length : 0;
        int overall_length = RESULT_SIZE + addm;
        overall_length += NUM_IR_STATISTICS;
        overall_length += NUM_WEIGHTED_IR_STATISTICS;
        overall_length += NUM_UNWEIGHTED_IR_STATISTICS;
        if (getAttributeID() >= 0) {
            overall_length += 1;
        }
        if (getPredTargetColumn()) {
            overall_length += 2;
        }

        overall_length += m_numPluginStatistics;

        String[] resultNames = new String[overall_length];
        int current = 0;
        resultNames[current++] = "Number_of_training_instances";
        resultNames[current++] = "Number_of_testing_instances";

        // Basic performance stats - right vs wrong
        resultNames[current++] = "Number_correct";
        resultNames[current++] = "Number_incorrect";
        resultNames[current++] = "Number_unclassified";
        resultNames[current++] = "Percent_correct";
        resultNames[current++] = "Percent_incorrect";
        resultNames[current++] = "Percent_unclassified";
        resultNames[current++] = "Kappa_statistic";

        // Sensitive stats - certainty of predictions
        resultNames[current++] = "Mean_absolute_error";
        resultNames[current++] = "Root_mean_squared_error";
        resultNames[current++] = "Relative_absolute_error";
        resultNames[current++] = "Root_relative_squared_error";

        // SF stats
        resultNames[current++] = "SF_prior_entropy";
        resultNames[current++] = "SF_scheme_entropy";
        resultNames[current++] = "SF_entropy_gain";
        resultNames[current++] = "SF_mean_prior_entropy";
        resultNames[current++] = "SF_mean_scheme_entropy";
        resultNames[current++] = "SF_mean_entropy_gain";

        // K&B stats
        resultNames[current++] = "KB_information";
        resultNames[current++] = "KB_mean_information";
        resultNames[current++] = "KB_relative_information";

        // IR stats
        resultNames[current++] = "True_positive_rate";
        resultNames[current++] = "Num_true_positives";
        resultNames[current++] = "False_positive_rate";
        resultNames[current++] = "Num_false_positives";
        resultNames[current++] = "True_negative_rate";
        resultNames[current++] = "Num_true_negatives";
        resultNames[current++] = "False_negative_rate";
        resultNames[current++] = "Num_false_negatives";
        resultNames[current++] = "IR_precision";
        resultNames[current++] = "IR_recall";
        resultNames[current++] = "F_measure";
        resultNames[current++] = "Matthews_correlation";
        resultNames[current++] = "Area_under_ROC";
        resultNames[current++] = "Area_under_PRC";

        // Weighted IR stats
        resultNames[current++] = "Weighted_avg_true_positive_rate";
        resultNames[current++] = "Weighted_avg_false_positive_rate";
        resultNames[current++] = "Weighted_avg_true_negative_rate";
        resultNames[current++] = "Weighted_avg_false_negative_rate";
        resultNames[current++] = "Weighted_avg_IR_precision";
        resultNames[current++] = "Weighted_avg_IR_recall";
        resultNames[current++] = "Weighted_avg_F_measure";
        resultNames[current++] = "Weighted_avg_matthews_correlation";
        resultNames[current++] = "Weighted_avg_area_under_ROC";
        resultNames[current++] = "Weighted_avg_area_under_PRC";

        // Unweighted IR stats
        resultNames[current++] = "Unweighted_macro_avg_F_measure";
        resultNames[current++] = "Unweighted_micro_avg_F_measure";

        // Timing stats
        resultNames[current++] = "Elapsed_Time_training";
        resultNames[current++] = "Elapsed_Time_testing";
        resultNames[current++] = "UserCPU_Time_training";
        resultNames[current++] = "UserCPU_Time_testing";
        resultNames[current++] = "UserCPU_Time_millis_training";
        resultNames[current++] = "UserCPU_Time_millis_testing";

        // sizes
        resultNames[current++] = "Serialized_Model_Size";
        resultNames[current++] = "Serialized_Train_Set_Size";
        resultNames[current++] = "Serialized_Test_Set_Size";

        // Prediction interval statistics
        resultNames[current++] = "Coverage_of_Test_Cases_By_Regions";
        resultNames[current++] = "Size_of_Predicted_Regions";

        // ID/Targets/Predictions
        if (getAttributeID() >= 0) {
            resultNames[current++] = "Instance_ID";
        }
        if (getPredTargetColumn()) {
            resultNames[current++] = "Targets";
            resultNames[current++] = "Predictions";
        }

        // Classifier defined extras
        resultNames[current++] = "Summary";
        // add any additional measures
        for (int i = 0; i < addm; i++) {
            resultNames[current++] = m_AdditionalMeasures[i];
        }

        for (AbstractEvaluationMetric m : m_pluginMetrics) {
            List<String> statNames = m.getStatisticNames();
            for (String s : statNames) {
                resultNames[current++] = s;
            }
        }

        if (current != overall_length) {
            throw new Error("ResultNames didn't fit RESULT_SIZE");
        }
        return resultNames;
    }

    /**
     * Gets the results for the supplied train and test datasets. Now performs a
     * deep copy of the classifier before it is built and evaluated (just in case
     * the classifier is not initialized properly in buildClassifier()).
     *
     * @param train the training Instances.
     * @param test  the testing Instances.
     * @return the results stored in an array. The objects stored in the array may
     *         be Strings, Doubles, or null (for the missing value).
     * @throws Exception if a problem occurs while getting the results
     */
    @Override
    public Object[] getResult(Instances train, Instances test) throws Exception {

        if (train.classAttribute().type() != Attribute.NOMINAL) {
            throw new Exception("Class attribute is not nominal!");
        }
        if (m_Template == null) {
            throw new Exception("No classifier has been specified");
        }
        int addm = (m_AdditionalMeasures != null) ? m_AdditionalMeasures.length : 0;
        int overall_length = RESULT_SIZE + addm;
        overall_length += NUM_IR_STATISTICS;
        overall_length += NUM_WEIGHTED_IR_STATISTICS;
        overall_length += NUM_UNWEIGHTED_IR_STATISTICS;
        if (getAttributeID() >= 0) {
            overall_length += 1;
        }
        if (getPredTargetColumn()) {
            overall_length += 2;
        }

        overall_length += m_numPluginStatistics;

        ThreadMXBean thMonitor = ManagementFactory.getThreadMXBean();
        boolean canMeasureCPUTime = thMonitor.isThreadCpuTimeSupported();
        if (canMeasureCPUTime && !thMonitor.isThreadCpuTimeEnabled()) {
            thMonitor.setThreadCpuTimeEnabled(true);
        }

        Object[] result = new Object[overall_length];
        Evaluation eval = new Evaluation(train);
        m_Classifier = AbstractClassifier.makeCopy(m_Template);
        double[] predictions;
        long thID = Thread.currentThread().getId();
        long CPUStartTime = -1, trainCPUTimeElapsed = -1, testCPUTimeElapsed = -1, trainTimeStart, trainTimeElapsed, testTimeStart, testTimeElapsed;

        // training classifier
        trainTimeStart = System.currentTimeMillis();
        if (canMeasureCPUTime) {
            CPUStartTime = thMonitor.getThreadUserTime(thID);
        }
        m_Classifier.buildClassifier(train);
        if (canMeasureCPUTime) {
            trainCPUTimeElapsed = thMonitor.getThreadUserTime(thID) - CPUStartTime;
        }
        trainTimeElapsed = System.currentTimeMillis() - trainTimeStart;

        // testing classifier
        testTimeStart = System.currentTimeMillis();
        if (canMeasureCPUTime) {
            CPUStartTime = thMonitor.getThreadUserTime(thID);
        }
        predictions = eval.evaluateModel(m_Classifier, test);
        if (canMeasureCPUTime) {
            testCPUTimeElapsed = thMonitor.getThreadUserTime(thID) - CPUStartTime;
        }
        testTimeElapsed = System.currentTimeMillis() - testTimeStart;
        thMonitor = null;

        m_result = eval.toSummaryString();
        // The results stored are all per instance -- can be multiplied by the
        // number of instances to get absolute numbers
        int current = 0;
        result[current++] = new Double(train.numInstances());
        result[current++] = new Double(eval.numInstances());
        result[current++] = new Double(eval.correct());
        result[current++] = new Double(eval.incorrect());
        result[current++] = new Double(eval.unclassified());
        result[current++] = new Double(eval.pctCorrect());
        result[current++] = new Double(eval.pctIncorrect());
        result[current++] = new Double(eval.pctUnclassified());
        result[current++] = new Double(eval.kappa());

        result[current++] = new Double(eval.meanAbsoluteError());
        result[current++] = new Double(eval.rootMeanSquaredError());
        result[current++] = new Double(eval.relativeAbsoluteError());
        result[current++] = new Double(eval.rootRelativeSquaredError());

        result[current++] = new Double(eval.SFPriorEntropy());
        result[current++] = new Double(eval.SFSchemeEntropy());
        result[current++] = new Double(eval.SFEntropyGain());
        result[current++] = new Double(eval.SFMeanPriorEntropy());
        result[current++] = new Double(eval.SFMeanSchemeEntropy());
        result[current++] = new Double(eval.SFMeanEntropyGain());

        // K&B stats
        result[current++] = new Double(eval.KBInformation());
        result[current++] = new Double(eval.KBMeanInformation());
        result[current++] = new Double(eval.KBRelativeInformation());

        // IR stats
        result[current++] = new Double(eval.truePositiveRate(m_IRclass));
        result[current++] = new Double(eval.numTruePositives(m_IRclass));
        result[current++] = new Double(eval.falsePositiveRate(m_IRclass));
        result[current++] = new Double(eval.numFalsePositives(m_IRclass));
        result[current++] = new Double(eval.trueNegativeRate(m_IRclass));
        result[current++] = new Double(eval.numTrueNegatives(m_IRclass));
        result[current++] = new Double(eval.falseNegativeRate(m_IRclass));
        result[current++] = new Double(eval.numFalseNegatives(m_IRclass));
        result[current++] = new Double(eval.precision(m_IRclass));
        result[current++] = new Double(eval.recall(m_IRclass));
        result[current++] = new Double(eval.fMeasure(m_IRclass));
        result[current++] = new Double(eval.matthewsCorrelationCoefficient(m_IRclass));
        result[current++] = new Double(eval.areaUnderROC(m_IRclass));
        result[current++] = new Double(eval.areaUnderPRC(m_IRclass));

        // Weighted IR stats
        result[current++] = new Double(eval.weightedTruePositiveRate());
        result[current++] = new Double(eval.weightedFalsePositiveRate());
        result[current++] = new Double(eval.weightedTrueNegativeRate());
        result[current++] = new Double(eval.weightedFalseNegativeRate());
        result[current++] = new Double(eval.weightedPrecision());
        result[current++] = new Double(eval.weightedRecall());
        result[current++] = new Double(eval.weightedFMeasure());
        result[current++] = new Double(eval.weightedMatthewsCorrelation());
        result[current++] = new Double(eval.weightedAreaUnderROC());
        result[current++] = new Double(eval.weightedAreaUnderPRC());

        // Unweighted IR stats
        result[current++] = new Double(eval.unweightedMacroFmeasure());
        result[current++] = new Double(eval.unweightedMicroFmeasure());

        // Timing stats
        result[current++] = new Double(trainTimeElapsed / 1000.0);
        result[current++] = new Double(testTimeElapsed / 1000.0);
        if (canMeasureCPUTime) {
            result[current++] = new Double((trainCPUTimeElapsed / 1000000.0) / 1000.0);
            result[current++] = new Double((testCPUTimeElapsed / 1000000.0) / 1000.0);

            result[current++] = new Double(trainCPUTimeElapsed / 1000000.0);
            result[current++] = new Double(testCPUTimeElapsed / 1000000.0);
        } else {
            result[current++] = new Double(Utils.missingValue());
            result[current++] = new Double(Utils.missingValue());
            result[current++] = new Double(Utils.missingValue());
            result[current++] = new Double(Utils.missingValue());
        }

        // sizes
        if (m_NoSizeDetermination) {
            result[current++] = -1.0;
            result[current++] = -1.0;
            result[current++] = -1.0;
        } else {
            ByteArrayOutputStream bastream = new ByteArrayOutputStream();
            ObjectOutputStream oostream = new ObjectOutputStream(bastream);
            oostream.writeObject(m_Classifier);
            result[current++] = new Double(bastream.size());
            bastream = new ByteArrayOutputStream();
            oostream = new ObjectOutputStream(bastream);
            oostream.writeObject(train);
            result[current++] = new Double(bastream.size());
            bastream = new ByteArrayOutputStream();
            oostream = new ObjectOutputStream(bastream);
            oostream.writeObject(test);
            result[current++] = new Double(bastream.size());
        }

        // Prediction interval statistics
        result[current++] = new Double(eval.coverageOfTestCasesByPredictedRegions());
        result[current++] = new Double(eval.sizeOfPredictedRegions());

        // IDs
        if (getAttributeID() >= 0) {
            String idsString = "";
            if (test.attribute(m_attID).isNumeric()) {
                if (test.numInstances() > 0) {
                    idsString += test.instance(0).value(m_attID);
                }
                for (int i = 1; i < test.numInstances(); i++) {
                    idsString += "|" + test.instance(i).value(m_attID);
                }
            } else {
                if (test.numInstances() > 0) {
                    idsString += test.instance(0).stringValue(m_attID);
                }
                for (int i = 1; i < test.numInstances(); i++) {
                    idsString += "|" + test.instance(i).stringValue(m_attID);
                }
            }
            result[current++] = idsString;
        }

        if (getPredTargetColumn()) {
            if (test.classAttribute().isNumeric()) {
                // Targets
                if (test.numInstances() > 0) {
                    String targetsString = "";
                    targetsString += test.instance(0).value(test.classIndex());
                    for (int i = 1; i < test.numInstances(); i++) {
                        targetsString += "|" + test.instance(i).value(test.classIndex());
                    }
                    result[current++] = targetsString;
                }

                // Predictions
                if (predictions.length > 0) {
                    String predictionsString = "";
                    predictionsString += predictions[0];
                    for (int i = 1; i < predictions.length; i++) {
                        predictionsString += "|" + predictions[i];
                    }
                    result[current++] = predictionsString;
                }
            } else {
                // Targets
                if (test.numInstances() > 0) {
                    String targetsString = "";
                    targetsString += test.instance(0).stringValue(test.classIndex());
                    for (int i = 1; i < test.numInstances(); i++) {
                        targetsString += "|" + test.instance(i).stringValue(test.classIndex());
                    }
                    result[current++] = targetsString;
                }

                // Predictions
                if (predictions.length > 0) {
                    String predictionsString = "";
                    predictionsString += test.classAttribute().value((int) predictions[0]);
                    for (int i = 1; i < predictions.length; i++) {
                        predictionsString += "|" + test.classAttribute().value((int) predictions[i]);
                    }
                    result[current++] = predictionsString;
                }
            }
        }

        if (m_Classifier instanceof Summarizable) {
            result[current++] = ((Summarizable) m_Classifier).toSummaryString();
        } else {
            result[current++] = null;
        }

        for (int i = 0; i < addm; i++) {
            if (m_doesProduce[i]) {
                try {
                    double dv = ((AdditionalMeasureProducer) m_Classifier).getMeasure(m_AdditionalMeasures[i]);
                    if (!Utils.isMissingValue(dv)) {
                        Double value = new Double(dv);
                        result[current++] = value;
                    } else {
                        result[current++] = null;
                    }
                } catch (Exception ex) {
                    System.err.println(ex);
                }
            } else {
                result[current++] = null;
            }
        }

        // get the actual metrics from the evaluation object
        List<AbstractEvaluationMetric> metrics = eval.getPluginMetrics();
        if (metrics != null) {
            for (AbstractEvaluationMetric m : metrics) {
                if (m.appliesToNominalClass()) {
                    List<String> statNames = m.getStatisticNames();
                    for (String s : statNames) {
                        result[current++] = new Double(m.getStatistic(s));
                    }
                }
            }
        }

        if (current != overall_length) {
            throw new Error("Results didn't fit RESULT_SIZE");
        }

        m_Evaluation = eval;

        return result;
    }

    /**
     * Returns the tip text for this property
     *
     * @return tip text for this property suitable for displaying in the
     *         explorer/experimenter gui
     */
    public String classifierTipText() {
        return "The classifier to use.";
    }

    /**
     * Get the value of Classifier.
     *
     * @return Value of Classifier.
     */
    public Classifier getClassifier() {

        return m_Template;
    }

    /**
     * Sets the classifier.
     *
     * @param newClassifier the new classifier to use.
     */
    public void setClassifier(Classifier newClassifier) {

        m_Template = newClassifier;
        updateOptions();
    }

    /**
     * Get the value of ClassForIRStatistics.
     *
     * @return Value of ClassForIRStatistics.
     */
    public int getClassForIRStatistics() {
        return m_IRclass;
    }

    /**
     * Set the value of ClassForIRStatistics.
     *
     * @param v Value to assign to ClassForIRStatistics.
     */
    public void setClassForIRStatistics(int v) {
        m_IRclass = v;
    }

    /**
     * Get the index of Attibute Identifying the instances
     *
     * @return index of outputed Attribute.
     */
    public int getAttributeID() {
        return m_attID;
    }

    /**
     * Set the index of Attibute Identifying the instances
     *
     * @param v index the attribute to output
     */
    public void setAttributeID(int v) {
        m_attID = v;
    }

    /**
     * @return true if the prediction and target columns must be outputed.
     */
    public boolean getPredTargetColumn() {
        return m_predTargetColumn;
    }

    /**
     * Set the flag for prediction and target output.
     *
     * @param v true if the 2 columns have to be outputed. false otherwise.
     */
    public void setPredTargetColumn(boolean v) {
        m_predTargetColumn = v;
    }

    /**
     * Returns whether the size determination (train/test/classifer) is skipped.
     *
     * @return true if size determination skipped
     */
    public boolean getNoSizeDetermination() {
        return m_NoSizeDetermination;
    }

    /**
     * Sets whether the size determination (train/test/classifer) is skipped.
     *
     * @param value true if to determine sizes
     */
    public void setNoSizeDetermination(boolean value) {
        m_NoSizeDetermination = value;
    }

    /**
     * Returns the tip text for this property
     *
     * @return tip text for this property suitable for displaying in the
     *         explorer/experimenter gui
     */
    public String noSizeDeterminationTipText() {
        return "If enabled, the size determination for train/test/classifier is skipped.";
    }

    /**
     * Updates the options that the current classifier is using.
     */
    protected void updateOptions() {

        if (m_Template instanceof OptionHandler) {
            m_ClassifierOptions = Utils.joinOptions(((OptionHandler) m_Template).getOptions());
        } else {
            m_ClassifierOptions = "";
        }
        if (m_Template instanceof Serializable) {
            ObjectStreamClass obs = ObjectStreamClass.lookup(m_Template.getClass());
            m_ClassifierVersion = "" + obs.getSerialVersionUID();
        } else {
            m_ClassifierVersion = "";
        }
    }

    /**
     * Set the Classifier to use, given it's class name. A new classifier will be
     * instantiated.
     *
     * @param newClassifierName the Classifier class name.
     * @throws Exception if the class name is invalid.
     */
    public void setClassifierName(String newClassifierName) throws Exception {

        try {
            setClassifier((Classifier) Class.forName(newClassifierName).newInstance());
        } catch (Exception ex) {
            throw new Exception("Can't find Classifier with class name: " + newClassifierName);
        }
    }

    /**
     * Gets the raw output from the classifier
     *
     * @return the raw output from th,0e classifier
     */
    @Override
    public String getRawResultOutput() {
        StringBuffer result = new StringBuffer();

        if (m_Classifier == null) {
            return "<null> classifier";
        }
        result.append(toString());
        result.append("Classifier model: \n" + m_Classifier.toString() + '\n');

        // append the performance statistics
        if (m_result != null) {
            result.append(m_result);

            if (m_doesProduce != null) {
                for (int i = 0; i < m_doesProduce.length; i++) {
                    if (m_doesProduce[i]) {
                        try {
                            double dv = ((AdditionalMeasureProducer) m_Classifier).getMeasure(m_AdditionalMeasures[i]);
                            if (!Utils.isMissingValue(dv)) {
                                Double value = new Double(dv);
                                result.append(m_AdditionalMeasures[i] + " : " + value + '\n');
                            } else {
                                result.append(m_AdditionalMeasures[i] + " : " + '?' + '\n');
                            }
                        } catch (Exception ex) {
                            System.err.println(ex);
                        }
                    }
                }
            }
        }
        return result.toString();
    }

    /**
     * Returns a text description of the split evaluator.
     *
     * @return a text description of the split evaluator.
     */
    @Override
    public String toString() {

        String result = "ClassifierSplitEvaluator: ";
        if (m_Template == null) {
            return result + "<null> classifier";
        }
        return result + m_Template.getClass().getName() + " " + m_ClassifierOptions + "(version " + m_ClassifierVersion + ")";
    }

}
